# %%
import pandas as pd
import pylab as plt
import seaborn as sbn
# %%
ls data
cat data/info.txt
UPLC_Plasma_Clinic = pd.read_csv("data/UPLC_Plasma_Clinic.txt",
sep='\t',
decimal=',',
index_col='Sample')
UPLC_Plasma_Clinic.head()
UPLC_Plasma_Clinic.replace({'CaseControl':
{0: 'control', 1:'case'}}
).head()
fg = sbn.FacetGrid(data=UPLC_Plasma_Clinic,
row='CaseControl',
col='Sex')
fg.map(plt.hist, "Age")
with plt.xkcd():
fg = sbn.FacetGrid(data=UPLC_Plasma_Clinic,
row='CaseControl',
col='Sex', size=3, aspect=2)
fg.map(plt.hist, "Age", histtype='stepfilled')
bootstrapping¶
we would like to verify how anomalous are certain "structures" that we observe.
One way of doing so it to use our data to recreate similar looking random data and see how rare it is to observe a specific difference.
male_mean_age = UPLC_Plasma_Clinic.query("Sex=='male'")['Age'].mean()
female_mean_age = UPLC_Plasma_Clinic.query("Sex=='female'")['Age'].mean()
female_mean_age - male_mean_age
how uncommon it would to see that difference if the distributions were the same?
UPLC_Plasma_Clinic['Age'].sample(n=5, replace=False)
clone = UPLC_Plasma_Clinic.copy()
clone['Age'] = clone['Age'].sample(n=len(clone), replace=True).values
clone_male_mean_age = clone.query("Sex=='male'")['Age'].mean()
clone_female_mean_age = clone.query("Sex=='female'")['Age'].mean()
clone_female_mean_age - clone_male_mean_age
We're missing something...what?
We should define this as a function, there is already enough clutter!
def sex_age_difference(df):
"""calculate the age differences between the genders"""
male_mean_age = df.query("Sex=='male'")['Age'].mean()
female_mean_age = df.query("Sex=='female'")['Age'].mean()
return female_mean_age - male_mean_age
def reshuffle_age(df):
"""create a cloned dataframe with reshuffled ages"""
clone = df.copy()
clone['Age'] = clone['Age'].sample(n=len(clone), replace=True).values
return clone
sex_age_difference(reshuffle_age(UPLC_Plasma_Clinic))
replicas = plt.array([sex_age_difference(reshuffle_age(UPLC_Plasma_Clinic))
for i in range(20)])
replicas[:3]
with plt.xkcd():
plt.hist(replicas, bins=20)
plt.axvline(sex_age_difference(UPLC_Plasma_Clinic), color='r')
sum(replicas<sex_age_difference(UPLC_Plasma_Clinic))/len(replicas)
to be able to replicate this result, we would have to "fix" the random numbers!
print(UPLC_Plasma_Clinic['Age'].sample().values)
print(UPLC_Plasma_Clinic['Age'].sample().values)
print(UPLC_Plasma_Clinic['Age'].sample().values)
print(UPLC_Plasma_Clinic['Age'].sample(random_state=1).values)
print(UPLC_Plasma_Clinic['Age'].sample(random_state=1).values)
print(UPLC_Plasma_Clinic['Age'].sample(random_state=1).values)
data transform¶
UPLC_Plasma_ExpDes = pd.read_csv("data/UPLC_Plasma_ExpDes.txt",
sep='\t',
decimal=',',
index_col='Sample')
UPLC_Plasma_ExpDes.head()
len(UPLC_Plasma_Clinic), len(UPLC_Plasma_ExpDes)
wait...
let's re-read the details of the data:
Replicated standards have "stand" label and
duplicated samples have "_D" label
so, if we ignore the replicated, we should have the same index, right?
UPLC_Plasma_ExpDes.drop(UPLC_Plasma_Clinic.index).head()
UPLC_Plasma_ExpDes.loc[UPLC_Plasma_Clinic.index].head()
shared_index = UPLC_Plasma_ExpDes.index.isin(UPLC_Plasma_Clinic.index)
UPLC_Plasma_ExpDes[~shared_index].head()
is_stand = UPLC_Plasma_ExpDes.index.str.startswith('stand')
is_duplicated = UPLC_Plasma_ExpDes.index.str.endswith('_D')
UPLC_Plasma_ExpDes[is_duplicated].head()
UPLC_Plasma_ExpDes[is_duplicated].index.str.replace('_D', '')
We need to start working with out transform to see what is going on.
There are two main families of transformations that one needs to understand:
- grouping
- joining
- pivoting
grouping divide the dataset in sub dataset based on certain properties, apply an operation and merge the result together (such as calculating the average age for each Sex).
joining merge two different tables.
pivoting is used to transform something like a tidy table in a more squared table. The inverse operation is called melt.
from R for data science
UPLC_Plasma_Clinic.join(UPLC_Plasma_ExpDes).head()
pd.merge(UPLC_Plasma_Clinic, UPLC_Plasma_ExpDes,
how='left',
left_index=True,
right_index=True,
).head()
joined = UPLC_Plasma_Clinic.join(UPLC_Plasma_ExpDes)
Grouping¶
Divide data in various subgroups, apply an operation to each one of the subgroups, merge the results together
joined.groupby(['Sex', 'Plate'])['CaseControl'].count()
joined.groupby(['Sex', 'Plate'])['CaseControl'].count().unstack()
Pivoting e melting¶
they are the less intuitive operations.
Needed to convert long tables into wide tables and back.
Together with the joins, they are the main methods to manipulate tidy data into the shape most appropriate for analysis.
Let's see few examples to clarify
fake_data = [('Jane', '2016/01/01', 10),
('Jane', '2017/01/01', 11),
('Jane', '2018/01/01', 12),
('John', '2016/01/01', 8),
#('John', '2017/01/01', 9), # this information is missing
('John', '2018/01/01', 10),
]
fake_data = pd.DataFrame(fake_data, columns=['name', 'date', 'value'])
fake_data
fake_data.pivot(index='name', columns='date', values='value')
fake_data.pivot_table(index='name',
columns='date',
values='value',
fill_value=0)
pivoted = fake_data.pivot_table(index='name',
columns='date',
values='value',
fill_value=0)
pivoted.reset_index().melt(id_vars='name')
In all honesty, melt is probably the least intuitive command of the whole pandas package...
fake_data = [('Jane', '2016/01/01', 10),
('Jane', '2017/01/01', 11),
('Jane', '2018/01/01', 12),
('John', '2016/01/01', 8),
('John', '2017/01/01', 11),
('John', '2017/01/01', 12),
('John', '2018/01/01', 10),
]
fake_data = pd.DataFrame(fake_data, columns=['name', 'date', 'value'])
fake_data
abbiamo valori multipli per alcuni giorni, bisogna decidere come aggregare questi valori (se usiamo pivot_table devono per forza essere numerici)
fake_data.pivot_table(index='name',
columns='date',
values='value',
aggfunc=plt.mean)
fake_data.pivot_table(index='name', columns='date', values='value', aggfunc=max)
A simplified version of pivot and melt are the stack and unstack, that works on the dataframe indices and columns.
fake_data = pd.DataFrame(plt.randn(4, 4),
index=[['x', 'x', 'y', 'y'], [1, 2, 1, 2]],
columns=[['a', 'a', 'b', 'b'], ['c', 'd', 'c', 'd']],
)
fake_data
fake_data.stack()
fake_data.unstack()
going back to our data...
pd.pivot_table(joined,
index='Plate',
columns='Sex',
values='CaseControl',
aggfunc=pd.Series.count,
margins=True,
)
pd.pivot_table(joined,
index='Plate',
columns='Sex',
values='CaseControl',
aggfunc=plt.mean,
)
pd.pivot_table(joined,
index='Plate',
columns=['Sex', 'CaseControl'],
values='Age',
aggfunc=pd.Series.count,
margins=True,
).astype(int)
joined.groupby(['Sex', 'Plate'])['Age'].aggregate([plt.mean, plt.std])
(joined.groupby(['Plate', 'Sex'])['Age'].
aggregate([plt.mean, plt.std]).unstack())
grouped = joined.groupby(['Sex', 'Plate'])['Age']
aggregated = grouped.aggregate([plt.mean, plt.std]).unstack(level=0)
aggregated.head()
aggregated['mean', 'female'] - aggregated['mean', 'male']
aggregated['mean'].head()
aggregated.xs('female', level='Sex', axis=1)
Assignment¶
- create a user on bitbucket
- create a private repository
- DO NOT INCLUDE THE DATA IN THE REPOSITORY!
include me and Lennart as partecipant (egiampieri)
check differencies on males and females for glycan levels
- DOT NOT USE INFORMATIONS ABOUT CASES AND CONTROLS